#include <fstream>
#include <algorithm>
#include <stdexcept>
#include <vector>
#include <deque>
#include <string>
#include <iostream>


#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/foreach.hpp>
#include <boost/regex.hpp>



#define DEBUG_LEVEL 6
#include <debug_func.h>

#include <open_fstream.h>
#include <portable_timer.h>
#include <progress_indicator.h>
#include <stlhelper.h>
#include <merge_unique.h>
#include <ultoa.h>
#include <print_error.h>
#include <open_stream_name_with_substitutions.h>
#include <seq_pairs.h>
#include <md4.h>
#include <digest.h>
#include "get_arg.h"


using std::vector;
using std::deque;
using std::ofstream;
using std::ifstream;
using std::ostream;
using std::string;
using std::runtime_error;
using std::cerr;
using std::cin;
using std::cout;
using std::istream;

struct t_digest_used
	: public t_digest
{
	unsigned used;
	t_digest_used(unsigned i = 0):used(i){}
	bool operator ==(const t_digest_used& o) const ;
	bool operator <(const t_digest_used& o) const ;
};
inline bool t_digest_used::operator ==(const t_digest_used& o) const
{
	return static_cast<const t_digest&>(*this) == static_cast<const t_digest&>(o);
}
inline bool t_digest_used::operator <(const t_digest_used& o) const
{
	return static_cast<const t_digest&>(*this) < static_cast<const t_digest&>(o);
}


struct op_merge_digest_counts
{
	void operator()(t_digest_used& a, const t_digest_used b)
	{
		a.used |= b.used;
	}
};

//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

//		Main

//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
using boost::regex;
using boost::match_results;
using boost::regex_search;


bool get_pattern(const std::string& str, boost::regex& pattern_match_regex, 
				 const std::string& to_regex, bool ignore_case,
				 std::string& match)
{
    match  = boost::regex_replace(str,
                             pattern_match_regex,
                             to_regex,
                             boost::format_no_copy | 
                             boost::format_first_only);
	if (ignore_case)
		boost::algorithm::to_lower(match);
	return match.length();
}


//________________________________________________________________________________________

//		open_output_file

//________________________________________________________________________________________
bool open_output_file(ofstream& os, const string& file_name)
{
	if (!file_name.length())   
		return false;
	namespace fs = boost::filesystem;
	if (fs::exists(file_name) && fs::is_directory(file_name))
		throw std::runtime_error(file_name + " already exists and is a directory not a file.");
	os.open(file_name.c_str());
	if (!os)
		throw std::runtime_error("Could not open " + file_name);
	return true;
}





//________________________________________________________________________________________

//		get_ids_from_file

//________________________________________________________________________________________
void get_ids_from_file(ifstream& ifs, const string& file_name, istream::pos_type file_size, 
					   ostream& summary, deque<t_digest_used>& id_digests,
   						unsigned file_num, t_program_args& args)
{
	args.VERBOSE() << "\tParsing ids from [" << file_name << "]...\n";
	boost::regex pattern_match_regex(args.regex);


	unsigned DOTS_SIZE = 100000;
	t_progress_indicator dots(args.VERBOSE(),  DOTS_SIZE);
	dots.use_timer();
	unsigned cnt_line = 0;

	string line;
	while (getline(ifs, line))
	{
		++cnt_line;
		++dots;

		// get matching from regular expression
		string match;
		if (!get_pattern(line, pattern_match_regex, args.to_regex, args.ignore_case, match))
			continue;

		// calculate and save hash
		id_digests.push_back(t_digest_used(file_num));
		
		md4_hash(match, id_digests.back());

		if (dots.value() % (DOTS_SIZE * 10) == 0)
		{
			istream::pos_type file_size_so_far = ifs.tellg();
			unsigned dots_needed = unsigned(dots.value() * 1.0 *
											file_size / file_size_so_far);
			dots_needed += DOTS_SIZE;
			dots.set_limit(dots_needed);
		}
	}


	dots.finish();
	
	args.VERBOSE(2) << "\t\t" << id_digests.size() << " identifiers read from [" << file_name << "].\n";
	summary << std::setw(10) << id_digests.size() << "\tidentifiers read from [" << file_name << "].\n";
	args.VERBOSE(2) << "\t\tSorting ID list...\n";
	sort(id_digests.begin(), id_digests.end());
	args.VERBOSE(2) << "\t\tRemoving duplicates...\n";
	remove_duplicates(id_digests);
	args.VERBOSE() << "\t\t" << id_digests.size() << " unique identifiers from [" << file_name << "].\n";
	summary << std::setw(10) << id_digests.size() << "\tunique identifiers from [" << file_name << "].\n";

}

//________________________________________________________________________________________

//		t_ostreams

//________________________________________________________________________________________
class t_ostreams
{
public:
	t_ostreams(t_program_args& args_)
		: 
		args(args_),
		cnt_shared(0),
		cnt_merged(0)
	{
		do_file[0] = open_output_file(file[0], args.file_1);
		do_file[1] = open_output_file(file[1], args.file_2);
		do_file_shared = open_output_file(file_shared, args.file_shared);
		do_file_merged = open_output_file(file_merged, args.file_merged);
		cnt[0] = 
		cnt[1] = 0;
	}
	t_program_args& args;
	bool	  do_file[2];
	bool	  do_file_shared;
	bool	  do_file_merged;
	ofstream  file[2];
	ofstream  file_shared;
	ofstream  file_merged;
	unsigned  cnt[2];
	unsigned  cnt_shared;
	unsigned  cnt_merged;
};
//________________________________________________________________________________________

//		t_counts

//________________________________________________________________________________________
class t_counts
{
public:
	t_counts(const deque<t_digest_used>& id_digests)
		: 
		cnt_shared(0),
		cnt_merged(0)
	{
		cnt[0] = 
		cnt[1] = 0;
		BOOST_FOREACH(const t_digest_used& d, id_digests) 
			if (d.used == 1)
				++cnt[0];
		BOOST_FOREACH(const t_digest_used& d, id_digests) 
			if (d.used == 2)
				++cnt[1];
		BOOST_FOREACH(const t_digest_used& d, id_digests) 
			if (d.used == 3)
				++cnt_shared;
		cnt_merged = id_digests.size();

	}
	unsigned  cnt[2];
	unsigned  cnt_shared;
	unsigned  cnt_merged;
};


//________________________________________________________________________________________

//		print_matching

//________________________________________________________________________________________
void print_matching(t_ostreams& strms, ifstream& ifs, const string& file_name, 
               istream::pos_type file_size, 
			   deque<t_digest_used>& id_digests, 
			   unsigned file_num, t_program_args& args)
{
	if (!strms.do_file[file_num - 1] && 
		!strms.do_file_merged &&
		!strms.do_file_shared)
		return;

	args.VERBOSE() << "\tPrinting results from [" << file_name << "]...\n";
	boost::regex pattern_match_regex(args.regex);


	// go back to start
	ifs.clear();
	ifs.seekg(0, ifstream::beg);


	unsigned DOTS_SIZE = 50000;
	t_progress_indicator dots(args.VERBOSE(),  DOTS_SIZE);
	dots.use_timer();
	unsigned cnt_line = 0;

	string line;
	while (getline(ifs, line))
	{
		++cnt_line;
		++dots;

		// get matching from regular expression
		string match;
		if (!get_pattern(line, pattern_match_regex, args.to_regex, args.ignore_case, match))
			continue;

		// calculate and save hash
		t_digest_used hash_value;
		md4_hash(match, hash_value);

		typedef deque<t_digest_used>::iterator ITER;
		ITER iter = lower_bound(id_digests.begin(), id_digests.end(), hash_value);
		assert(iter != id_digests.end());

		// ignore if already printed: must be duplicate
		if (!iter->used)
			continue;


		// just me
		if (iter->used == file_num)
		{
			if (strms.do_file[file_num - 1])
				strms.file[file_num - 1] << line << "\n";
			++strms.cnt[file_num - 1];
		}

		// both
		if (iter->used == 3)
		{
			if (strms.do_file_shared)
				strms.file_shared << line << "\n";
			++strms.cnt_shared;
		}

		// either
		if (strms.do_file_merged)
			strms.file_merged << line << "\n";
		++strms.cnt_merged;

		// clear
		iter->used = 0;

		if (dots.value() % (DOTS_SIZE * 10) == 0)
		{
			istream::pos_type file_size_so_far = ifs.tellg();
			unsigned dots_needed = unsigned(dots.value() * 1.0 *
											file_size / file_size_so_far);
			dots_needed += DOTS_SIZE;
			dots.set_limit(dots_needed);
		}
	}
}

//________________________________________________________________________________________

//		print_summary

//________________________________________________________________________________________
void print_summary(t_counts& cnts, const string& file_name1, const string& file_name2, 
				   ofstream& summary, t_program_args& args)
{
		//
	// print results common to all
	//
	if (cnts.cnt[0])
		args.VERBOSE() << "\t\t" << cnts.cnt[0]
									<< "\tmatching identifiers only found in [" << file_name1 << "]\n";
	summary << std::setw(10)  << cnts.cnt[0]
									<< "\tmatching identifiers only found in [" << file_name1 << "]\n";
	if (cnts.cnt[1])
		args.VERBOSE() << "\t\t" << cnts.cnt[1]
									<< "\tmatching identifiers only found in [" << file_name2 << "]\n";
	summary << std::setw(10)  << cnts.cnt[1]                            
									<< "\tmatching identifiers only found in [" << file_name2 << "]\n";

	if (cnts.cnt_shared)
		args.VERBOSE() << "\t\t" << cnts.cnt_shared
									<< "\tmatching identifiers found in both files\n";
	summary << std::setw(10)  << cnts.cnt_shared
									<< "\tmatching identifiers found in both files\n";

	if (cnts.cnt_merged)
		args.VERBOSE() << "\t\t" << cnts.cnt_merged
									<< "\tmatching identifiers found in one file or the other\n";
	summary << std::setw(10)  << cnts.cnt_merged
									<< "\tmatching identifiers found in one file or the other\n";

}

//________________________________________________________________________________________

//		main

//________________________________________________________________________________________
int main(int argc, char* argv[])
{
	t_program_args args;

	try
	{

	debug_func_cerr(5);

	// don't use C streams
	std::ios::sync_with_stdio(false);


	// get command line arguments
	if (!process_command_line_options(argc, argv, args))
		return 1;


	//
	//	prepare summary stream
	//
	ofstream summary;
	if (!args.file_summary.empty())
	{
		summary.open(args.file_summary.c_str());
		if (!summary.good())
			throw std::runtime_error("Could not open " + args.file_summary);
	}

	//
	//	Get IDs from input file1
	//
	deque<t_digest_used> id_digests;
	ifstream input_fs1;
	string short_file_name1 = get_file_name_substitutions("[FILENAME][EXT]", args.file_input1);
	istream::pos_type file_size1 = open_fstream(input_fs1, args.file_input1);
	get_ids_from_file(input_fs1, short_file_name1, file_size1, summary, id_digests, 1, args);


	//
	//	Get IDs from input file2
	//
	deque<t_digest_used> id_digests2;
	ifstream input_fs2;
	string short_file_name2 = get_file_name_substitutions("[FILENAME][EXT]", args.file_input2);
	istream::pos_type file_size2 = open_fstream(input_fs2, args.file_input2);
	get_ids_from_file(input_fs2, short_file_name2, file_size2, summary, id_digests2, 2, args);


	//
	// merge together
	//
	id_digests.insert(id_digests.end(), id_digests2.begin(),  id_digests2.end());
	deque<t_digest_used>().swap(id_digests2);
	args.VERBOSE() << "\tSorting ids...\n";
	sort(id_digests.begin(), id_digests.end());
	args.VERBOSE() << "\tMerging duplicate ids from both files...\n";
	merge_unique(id_digests, std::equal_to<t_digest>(), op_merge_digest_counts());

	t_counts counts(id_digests);

	//
	//	print matching
	//
	t_ostreams strms (args);
	print_matching(strms, input_fs1, short_file_name1, file_size1, id_digests, 1, args);
	print_matching(strms, input_fs2, short_file_name2, file_size2, id_digests, 2, args);


	//
	//	print summary
	//
	print_summary(counts, short_file_name1, short_file_name2, summary, args);


	args.VERBOSE() << "\tAll finished.\n";

	}
	catch (std::exception& e)
	{
		std_print_error(args.CERR(), e.what());
		return 1;
	}

	return 0;
}

